#-*- coding = utf-8 -*-
#@Time : 21-3-3 下午6:03
#@Author : wei
#@File : spider9.py
#@Software: PyCharm


from lxml import etree
import re
import requests
from pymongo import MongoClient


class Doubanspider():
    def __init__(self):
        self.url = 'https://movie.douban.com/top250?start={}'
        self.header = {
            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36"
            }

    #构造url列表
    def get_url_list(self):
        return [self.url.format(page*25) for page in range(0,10)]

    #发送请求，获取响应
    def parse_url(self,url):
        return requests.get(url,headers = self.header).text

    #提取所需数据
    def get_data_list(self,html_str):
        html = etree.HTML(html_str)
        datalist = []
        li_list = html.xpath('//ol[@class="grid_view"]/li')
        for li in li_list:
            data = []
            link = li.xpath('./div/div/a/@href')[0]
            #print(link)
            data.append(link)
            imgSrc = li.xpath('./div/div/a/img/@src')[0]
            data.append(imgSrc)
            titles = li.xpath('./div//span[@class="title"]/text()')
            #print(titles)
            if len(titles)==2:
                data.append(titles[0])
                data.append(titles[1].replace('/', ''))
            else:
                data.append(titles[0])
                data.append(' ')
            rate = li.xpath('./div//span[@class="rating_num"]/text()')[0]
            data.append(rate)
            judgeNum = li.xpath('.//div[@class="star"]/span[4]/text()')[0]
            #print(judgeNum)
            judgeNum = re.findall('(\d+)',judgeNum)[0]
            data.append(judgeNum)
            inq = li.xpath('.//div[@class="bd"]/p[2]/span/text()')
            inq = inq[0].replace('。','')
            #print(inq)
            data.append(inq)
            bd = li.xpath('.//div[@class="bd"]/p/text()')[0]
            bd = re.sub('/',' ',bd)
            bd = re.sub(r'</br(\s+)?>(\s+)?',' ',bd)
            bd = re.sub(r'<br/(\s+)?>(\s+)?',' ',bd)
            data.append(bd)
            datalist.append(data)
        #print(datalist)
        return datalist

    #保存到mongodb数据库
    def save_to_mongo(self,datalist):
        client = MongoClient('mongodb://127.0.0.1:27017')
        db = client['doubanmovie']
        for data in datalist:
            db.movie.insert_one({
                            'info_link': data[0],
                            'img_link':data[1],
                            'cname': data[2],
                            'oname':data[3],
                            'rate':data[4],
                            'judge_number':data[5],
                            'instroduction':data[6],
                            'info' : data[7]})
        print('saveing...')

    #实现主要逻辑
    def run(self):
        #构造url列表
        url_list = self.get_url_list()
        #遍历，发送请求，获取响应
        for url in url_list:
            html_str = self.parse_url(url)
            datalist = self.get_data_list(html_str)
            #保存
            self.save_to_mongo(datalist)

if __name__ == '__main__':
    doubanmovie = Doubanspider()
    doubanmovie.run()